package com.coderfans.pageprocessor.pageprocessor;

import com.coderfans.pageprocessor.SinglePageModel;
import com.coderfans.redis.RedisKey;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.xsoup.Xsoup;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Date;
import java.util.List;

/**
 * Created by ${fanchunshuai} on 2016-7-27.
 *
 * @version: V1.0
 * @Desc: github网页下载测试
 * @Copyright (c) 2016 58到家-版权所有
 */
public class GithubPageDownloadProcessor implements PageProcessor {

    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    int i = 0;

    //定制爬虫逻辑的核心接口，这里编写抽取逻辑
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+)").all());
        if(StringUtils.isNotBlank(page.getHtml().get())){
            String filename = page.getRequest().getUrl().replace(":","").replace("/","-");
            System.out.println("filename = "+filename);
            String filepath = "E:/githubfilelist/"+String.format("yyyy-MM-dd",new Date())+"/"+filename+".html";
            File file = new File(filepath);
            if (!file.getParentFile().exists()) {
                file.getParentFile().mkdirs();
            }
            try {
                if(!file.exists()){
                    file.createNewFile();
                    FileWriter fw = new FileWriter(file, true);
                    BufferedWriter bw = new BufferedWriter(fw);
                    bw.write(page.getHtml().get());
                    bw.flush();
                    bw.close();
                    fw.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        /*if (page.getResultItems().get("name")==null){
            //设置skip之后，这个页面的结果不会被Pipeline处理
            page.setSkip(true);
        }*/
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
/*
        Spider.create(new GithubPageDownloadProcessor()).addPipeline(new com.coderfans.datagithub.ConsolePipeline())
                .addUrl("https://github.com/lw-lin")
                .thread(8).run();
*/
        String rootPath = "E:/githubfileslist";



        try {
            //组织页面中的数据
            Document document = Jsoup.parse(new File("E:\\githubfileslist\\URLGITHUB_TEST_LIST\\https-githubcomperka.html"),"UTF-8");
            List<String> hrefList = Xsoup.compile("//a/@href").evaluate(document).list();
            for (String str : hrefList){

                /*String orgNum = Xsoup.compile("//nav[@class='orgnav']/a//span[@class='counter']/text()").evaluate(document).get();
                System.out.println("orgnum = "+orgNum);
                int orgUserNum  = Integer.parseInt(orgNum);
                int pageNum = 0;
                if(orgUserNum > 0){
                    pageNum = orgUserNum / 30;  //默认每页用户30个
                    if(orgUserNum % 30 != 0){
                        pageNum = pageNum + 1;
                    }
                }*/

                String urlArray [] = str.split("/");
                String fileName = "";
                if(urlArray.length >3){
                    fileName = (urlArray[0]+urlArray[2]+urlArray[3]).replace(":","--").replace(".","");
                }else {
                    fileName = (urlArray[0]+urlArray[2]).replace(":","--").replace(".","");
                }
                //对文件名进行过滤
                if (fileName.contains("%") || fileName.contains("?")){
                    continue;
                }
                System.out.println(fileName+"----------------------");
                SinglePageModel singlePageModel = new SinglePageModel(rootPath,RedisKey.GITHUB_URL_TEST_LIST_ROOTPATH,fileName);
                singlePageModel.dowloadSpider(str,4);
            }
            /*String orgNum = Xsoup.compile("//nav[@class='orgnav']/a//span[@class='counter']/text()").evaluate(document).get();
            System.out.println("orgnum = "+orgNum);
            String url = "https://github.com/perka";
            int num  = Integer.parseInt(orgNum);
            String fileName1 = "";
            for (int i = 0; i < 1 ;i++){
                String urlArray [] = url.split("/");
                String newUrl = urlArray[0]+"//"+urlArray[2]+"/orgs/"+urlArray[3]+"/people?page="+(i+1);
                String fileName = (urlArray[0]+urlArray[2]+urlArray[3]).replace(":","-").replace(".","");
                System.out.println(fileName+"----------------------");
                fileName1 = fileName;
                SinglePageModel singlePageModel = new SinglePageModel(rootPath,RedisKey.GITHUB_URL_TEST_LIST_ROOTPATH,fileName);
                singlePageModel.dowloadSpider(newUrl,4);
            }
*/

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
